/*
 * Copyright (C) 1994-2021 Altair Engineering, Inc.
 * For more information, contact Altair at www.altair.com.
 *
 * This file is part of both the OpenPBS software ("OpenPBS")
 * and the PBS Professional ("PBS Pro") software.
 *
 * Open Source License Information:
 *
 * OpenPBS is free software. You can redistribute it and/or modify it under
 * the terms of the GNU Affero General Public License as published by the
 * Free Software Foundation, either version 3 of the License, or (at your
 * option) any later version.
 *
 * OpenPBS is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
 * License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 * Commercial License Information:
 *
 * PBS Pro is commercially licensed software that shares a common core with
 * the OpenPBS software.  For a copy of the commercial license terms and
 * conditions, go to: (http://www.pbspro.com/agreement.html) or contact the
 * Altair Legal Department.
 *
 * Altair's dual-license business model allows companies, individuals, and
 * organizations to create proprietary derivative works of OpenPBS and
 * distribute them - whether embedded or bundled with other software -
 * under a commercial license agreement.
 *
 * Use of Altair's trademarks, including but not limited to "PBS™",
 * "OpenPBS®", "PBS Professional®", and "PBS Pro™" and Altair's logos is
 * subject to Altair's trademark licensing policies.
 */

/**
 * @file	qsub.c
 * @brief
 *	qsub - (PBS) submit batch job
 *
 * @author Terry Heidelberg
 *         Livermore Computing
 *
 * @author Bruce Kelly
 *         National Energy Research Supercomputer Center
 *
 * @author Lawrence Livermore National Laboratory
 *         University of California
 */

/**
 * @file    qsub.c
 *
 * @brief
 * qsub now has two components:
 * A forground process and a background process.
 * - The background process is loaded initially (per user, per target server)
 *   by the foreground.
 * - The background process reuses a authenticated server connection.
 * - The foreground process sends job information to background process
 *   which in turn communicates over the already established connection to the
 *   server. It returns back any jobid or error string (and code) to the
 *   foreground process.
 * - The background process quits silently if:
 *    a) The connection to the server is lost
 *    b) There are no requests sent to it for the last 1 minute.
 *
 */

#include <pbs_config.h> /* the master config generated by configure */
#include <pbs_version.h>

#include <sys/types.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <sys/utsname.h>
#include <sys/wait.h>
#include <netinet/in.h>
#include <errno.h>
#include <fcntl.h>
#include <netdb.h>
#include <signal.h>
#include <termios.h>
#include <assert.h>
#include <sys/un.h>
#include <syslog.h>
#include "pbs_ifl.h"
#include "cmds.h"
#include "libpbs.h"
#include "net_connect.h"
#include "dis.h"
#include "port_forwarding.h"
#include "credential.h"
#include "ticket.h"
#include "portability.h"

#ifdef LOG_BUF_SIZE
/* Also defined in port_forwarding.h */
#undef LOG_BUF_SIZE
#endif

#define LOG_BUF_SIZE 1024
#define ENV_PBS_JOBID "PBS_JOBID"
#define CMDLINE 3

#undef DEBUG
#undef DBPRT
#ifdef DEBUG
#define DBPRT(x) printf x;
#else
#define DBPRT(x)
#endif

#if defined(HAVE_SYS_IOCTL_H)
#include <sys/ioctl.h>
#endif /* HAVE_SYS_IOCTL_H */

#if defined(FD_SET_IN_SYS_SELECT_H)
#include <sys/select.h>
#endif

/*
 * For the purpose of unit testing, qsub is capable of printing a backtrace
 * when it exits with a non-zero status. This behavior is limited to systems
 * that support the backtrace(3) library function. To enable this behavior,
 * define the BACKTRACE_SIZE macro immediately following this comment. The
 * value assigned to BACKTRACE_SIZE is an integer that defines the maximum
 * depth of the backtrace. For example:
 * #define BACKTRACE_SIZE 100
 */
#ifdef BACKTRACE_SIZE
#include <execinfo.h>
#endif

#define MAX_QSUB_PREFIX_LEN 32
#define DMN_REFUSE_EXIT 7 /* return code when daemon can't serve a job and exits */

extern char *msg_force_qsub_update;

#define PBS_DPREFIX_DEFAULT "#PBS"
#define PBS_O_ENV "PBS_O_" /* prefix for environment variables created by qsub */

/* Warning/Error messages */
#define INTER_GUI_WARN "qsub: only interactive jobs can have GUI\n"
#define INTER_BLOCK_WARN "qsub (Warning) : setting \"block\" attribute as \"true\"" \
			 " for an interactive job will not return job's exit status\n"
#define INTER_ARRAY "qsub: interactive and array job submission cannot be used together\n"
#define NO_RERUN_ARRAY "qsub:  cannot submit non-rerunable Array Job\n"
#define INTER_RERUN_WARN "qsub (Warning): Interactive jobs will be treated as not rerunnable\n"
#define BAD_W "qsub: illegal -W value\n"
#define MULTIPLE_MAX_RUN "qsub: multiple max_run_subjobs values found\n"

/* Security library variables */
static int cs_init = 0; /*1 == security library initialized, 0 == not initialized*/
static int cred_type = -1;
size_t cred_len = 0;
char *cred_buf = NULL;
char cred_name[32];  /* space to hold small credential name */
char *tmpdir = NULL; /* Path of temp directory in which to put the job script */

/* variables for Interactive mode */
int comm_sock; /* Socket for interactive and block job */

#define X11_MSG_OFFSET sizeof(XAUTH_ERR_REDIRECTION) /* offset of the redirection clause */

extern char fl[];
char retmsg[MAXPATHLEN];				 /* holds the message that background qsub process will send */
char qsub_cwd[MAXPATHLEN + 1];				 /* buffer to pass cwd to background qsub */
char *new_jobname = NULL;				 /* return from submit request */
char destination[PBS_MAXDEST];				 /* Destination of the batch job, specified by q opt */
char server_out[PBS_MAXSERVERNAME + PBS_MAXPORTNUM + 2]; /* Destination server, parsed from destination[] */
char script_tmp[MAXPATHLEN + 1] = {'\0'};		 /* name of script file copy */
int sd_svr;						 /* return from pbs_connect */
char *display;						 /* environment variable DISPLAY */
struct attrl *attrib = NULL;				 /* Attribute list */
static struct attrl *attrib_o = NULL;			 /* Original attribute list, before applying default_qsub_arguments */
static char dir_prefix[MAX_QSUB_PREFIX_LEN + 1];	 /* Directive Prefix, specified by C opt */
static struct batch_status *ss = NULL;
static char *dfltqsubargs = NULL;			/* Default qsub arguments */
static char *pbs_hostvar = NULL;			/* buffer containing ",PBS_O_HOST=" and host name */
static int pbs_o_hostsize = sizeof(",PBS_O_HOST=") + 1; /* size of prefix for hostvar */

int pid = -1;

/*
 * Flag to check if current process is the background process.
 * This variable is set only once and is read-only afterwards.
 */
int is_background = 0;
char *basic_envlist = NULL;    /* basic comma-separated environment variables list string */
char *qsub_envlist = NULL;     /* comma-separated variables list string */
char *v_value = NULL;	       /* expanded variable list from v opt */
static int no_background = 0;  /* flag to disable backgrounding */
static char roptarg = 'y';     /* whether the job is rerunnable */
static char *v_value_o = NULL; /* copy of v_value before set_job_env() */
static int x11_disp = FALSE;   /* whether DISPLAY environment variable is available */

/* state booleans for protecting already-set options */
static int a_opt = FALSE;
static int c_opt = FALSE;
static int e_opt = FALSE;
static int h_opt = FALSE;
static int j_opt = FALSE;
static int k_opt = FALSE;
static int l_opt = FALSE;
static int m_opt = FALSE;
static int o_opt = FALSE;
static int p_opt = FALSE;
static int q_opt = FALSE;
static int r_opt = FALSE;
static int u_opt = FALSE;
static int v_opt = FALSE;
static int z_opt = FALSE;
static int A_opt = FALSE;
static int C_opt = FALSE;
static int J_opt = FALSE;
static int M_opt = FALSE;
static int N_opt = FALSE;
static int P_opt = FALSE;
static int R_opt = FALSE;
static int S_opt = FALSE;
static int V_opt = FALSE;
static int Depend_opt = FALSE;
static int Stagein_opt = FALSE;
static int Stageout_opt = FALSE;
static int Sandbox_opt = FALSE;
static int Grouplist_opt = FALSE;
static int Resvstart_opt = FALSE;
static int Resvend_opt = FALSE;
static int pwd_opt = FALSE;
static int cred_opt = FALSE;
static int block_opt = FALSE;
static int relnodes_on_stageout_opt = FALSE;
static int tolerate_node_failures_opt = FALSE;
static int roptarg_inter = FALSE;
int Interact_opt = FALSE;
int Forwardx11_opt = FALSE;
int gui_opt = FALSE;

/* for saving option booleans */
static int a_opt_o = FALSE;
static int c_opt_o = FALSE;
static int e_opt_o = FALSE;
static int h_opt_o = FALSE;
static int j_opt_o = FALSE;
static int k_opt_o = FALSE;
static int l_opt_o = FALSE;
static int m_opt_o = FALSE;
static int o_opt_o = FALSE;
static int p_opt_o = FALSE;
static int q_opt_o = FALSE;
static int r_opt_o = FALSE;
static int u_opt_o = FALSE;
static int v_opt_o = FALSE;
static int z_opt_o = FALSE;
static int A_opt_o = FALSE;
static int C_opt_o = FALSE;
static int J_opt_o = FALSE;
static int M_opt_o = FALSE;
static int N_opt_o = FALSE;
static int P_opt_o = FALSE;
static int S_opt_o = FALSE;
static int V_opt_o = FALSE;
static int Depend_opt_o = FALSE;
static int Interact_opt_o = FALSE;
static int Stagein_opt_o = FALSE;
static int Stageout_opt_o = FALSE;
static int Sandbox_opt_o = FALSE;
static int Grouplist_opt_o = FALSE;
static int gui_opt_o = FALSE;
static int Resvstart_opt_o = FALSE;
static int Resvend_opt_o = FALSE;
static int pwd_opt_o = FALSE;
static int cred_opt_o = FALSE;
static int block_opt_o = FALSE;
static int relnodes_on_stageout_opt_o = FALSE;
static int tolerate_node_failures_opt_o = FALSE;
static int max_run_opt = FALSE;

extern char **environ;

extern void blockint(int sig);
extern void do_daemon_stuff();
extern void enable_gui(void);
extern void set_sig_handlers(void);
extern void interactive(void);
extern int dorecv(void *, char *, int);
extern int dosend(void *, char *, int);
extern int daemon_submit(int *, int *);
extern int get_script(FILE *, char *, char *);
extern int check_for_background(int, char **);

void exit_qsub(int exitstatus);

/* The following are "Utility" functions. */

/**
 * @brief
 * 	Process comma separated tokens with consideration for quotes.
 *
 * @param[in]	str	source string to scan for tokens
 *
 * @retval	NULL	no more tokens
 *
 */
static char *
comma_token(char *str)
{
	static char *p = NULL;
	char quote = 0;
	char *tok;

	if (str != NULL)
		p = str;

	/* check for no more tokens */
	if ((p == NULL) || (*p == 0))
		return NULL;

	tok = p;
	for (; *p != '\0'; p++) {
		switch (*p) {

			case '\'':
			case '"':
				if (*p == quote) /* ending quote */
					quote = 0;
				else /* starting quote */
					quote = *p;
				break;

			case ',':
				if (quote == 0) { /* normal comma */
					*p++ = 0; /* terminate token */
					return tok;
				}
				break; /* comma inside quotes, keep scanning */

			case ESC_CHAR:		   /* pass over next char */
				if (*(p + 1) != 0) /* check '\' is not last */
					p++;
				break;
		}
	}
	return tok;
}

/**
 * @brief
 *      Copy an environment variable to a specified location
 *
 * @param[in]	dest	- The destination address
 * @param[in]   pv	- The source address
 * @param[in]   quote_flg - Whether quote characters should be escaped
 *
 * @return	char*
 * @retval	NULL - Failure
 * @retval	!NULL - Success - Pointer to pv parameter
 */
static char *
copy_env_value(char *dest, char *pv, int quote_flg)
{
	int go = 1;
	int q_ch = 0;
	int is_func = 0;
	char *dest_full = dest;

	while (*dest)
		++dest;

	is_func = ((*pv == '(') && (*(pv + 1) == ')') && (*(pv + 2) == ' ') && (*(pv + 3) == '{'));

	/*
	 * Keep the list of special characters consistent with encode_arst_bs()
	 * and parse_comma_string_bs().
	 */

	while (go && *pv) {
		switch (*pv) {
			case '"':
			case '\'':
				if (q_ch) { /* local quoting is in progress */
					if (q_ch == (int) *pv) {
						q_ch = 0; /* end quote */
					} else {
						*dest++ = ESC_CHAR; /* escape quote */
						*dest++ = *pv;
					}
				} else if (quote_flg) {	    /* global quoting is on */
					*dest++ = ESC_CHAR; /* escape quote */
					*dest++ = *pv;
				} else {
					q_ch = (int) *pv; /* turn local quoting on */
				}
				break;

			case ESC_CHAR: /* backslash in value, escape it */
				*dest++ = *pv;
				if (*(pv + 1) != ',') /* do not escape if ESC_CHAR already escapes */
					*dest++ = *pv;
				break;

			case ',':
				if (q_ch || quote_flg) {
					*dest++ = ESC_CHAR;
					*dest++ = *pv;
				} else if (dest_full != dest && *(dest - 1) == ESC_CHAR) { /* the comma is escaped, not finished yet */
					*dest++ = *pv;
				} else {
					go = 0; /* end of value string */
				}
				break;

			case ';':
				*dest++ = *pv;
				if (is_func && (*(pv + 1) == '\n'))
					pv++;
				break;

			default:
				*dest++ = *pv;
				break;
		}
		pv++;
	}

	*dest = '\0';
	if (q_ch)
		return NULL; /* error-unterminated quote */
	else
		return (pv);
}

/**
 * @brief
 *	Given a comma-separated list of "variable" or "variable=value"
 *	entries, return a new variable list with those "variable" entries
 *	expanded to contain their values obtained from the current
 *	environment.
 *
 * @param[in] varlist - variable list
 *
 * @return char *
 *	The malloced list of expanded variables list.
 *	NULL if any error encountered.
 *
 */
static char *
expand_varlist(char *varlist)
{
	char *v_value1 = NULL;
	char *v_value2 = NULL;
	char *vn = NULL;
	char *vv = NULL;
	char *p1, *p2, *p;
	char *ev;
	char *ev2;
	int v_value1_sz = 0;
	char *pc;
	int special_char_cnt = 0;
	int len = 0;
	int esc_char_cnt = 0;

	/*
	 * count special characters as they are escaped with '\' in copy_env_value function
	 * so that this is useful while calculating the accurate size of the destination string.
	 * Also calculating the length of the string.
	 */
	pc = varlist;
	for (; *pc; pc++) {
		if ((*pc == '"') || (*pc == '\'') || (*pc == ',') || (*pc == '\\'))
			special_char_cnt++;
		len++;
	}

	v_value1_sz = len + special_char_cnt + 1;
	/* final copy */
	v_value1 = malloc(v_value1_sz);
	if (v_value1 == NULL) {
		fprintf(stderr, "qsub: out of memory\n");
		return NULL;
	}
	v_value1[0] = '\0';

	/* working copy */
	v_value2 = strdup(varlist);
	if (v_value2 == NULL) {
		fprintf(stderr, "qsub: out of memory\n");
		goto expand_varlist_err;
	}

	p1 = comma_token(v_value2);
	while (p1 != NULL) {
		vn = p1;
		vv = NULL;
		if ((p2 = strchr(p1, '=')) != NULL) {
			*p2 = '\0';
			vv = p2 + 1;
		}
		if ((vv == NULL) && (strncmp(vn, PBS_O_ENV, sizeof(PBS_O_ENV) - 1) != 0)) {
			/* do not add PBS_O_* env variables, as these */
			/* are set by qsub */

			ev = getenv(vn);
			if (ev == NULL) {
				fprintf(stderr, "qsub: cannot send environment with the job\n");
				goto expand_varlist_err;
			}
			/* count escape characters as they are escaped with '\'*/
			ev2 = ev;
			len = 0;
			for (; *ev2; ev2++) {
				if ((*ev2 == ESC_CHAR))
					esc_char_cnt++;

				len++;
			}
			v_value1_sz = v_value1_sz + len + esc_char_cnt + 1; /* include '=' */
			p = realloc(v_value1, v_value1_sz);
			if (p == NULL) {
				fprintf(stderr, "qsub: out of memory\n");
				goto expand_varlist_err;
			}
			v_value1 = p;
			if (v_value1[0] != '\0')
				strcat(v_value1, ",");
			strcat(v_value1, vn);
			strcat(v_value1, "=");

			if (copy_env_value(v_value1, ev, 1) == NULL) {
				fprintf(stderr, "qsub: cannot send environment with the job\n");
				goto expand_varlist_err;
			}
		} else if (vv != NULL) {
			/* no need to adjust */
			if (v_value1[0] != '\0')
				strcat(v_value1, ",");
			strcat(v_value1, vn);
			strcat(v_value1, "=");
			if (copy_env_value(v_value1, vv, 0) == NULL) {
				fprintf(stderr, "qsub: cannot send environment with the job\n");
				goto expand_varlist_err;
			}
		}

		p1 = comma_token(NULL);
	}
	free(v_value2);
	return (v_value1);

expand_varlist_err:
	free(v_value1);
	free(v_value2);
	return NULL;
}

/**
 * @brief
 *	Query the server for a new value to "default_qsub_arguments".
 * @note
 *	References the global variables 'sd_svr', 'ss', and "dfltqsubargs'.
 *	'dfltqsubargs' is updated with the new value.
 *
 */
static void
refresh_dfltqsubargs(void)
{
	struct attrl *attr;
	struct batch_status *ss_save = NULL;
	char *errmsg;

	if (sd_svr == -1)
		return;

	free(dfltqsubargs);

	dfltqsubargs = NULL;
	ss = pbs_statserver(sd_svr, NULL, NULL);

	if (ss == NULL && pbs_errno != PBSE_NONE) {
		if ((errmsg = pbs_geterrmsg(sd_svr)) != NULL)
			fprintf(stderr, "qsub: %s\n", errmsg);
		else
			fprintf(stderr, "qsub: Error %d\n", pbs_errno);
		return;
	}

	ss_save = ss;
	while (ss != NULL) {
		for (attr = ss->attribs; attr != NULL; attr = attr->next) {
			if (strcmp(attr->name, ATTR_dfltqsubargs) == 0) {
				dfltqsubargs = strdup(attr->value);
				break;
			}
		}
		ss = ss->next;
	}
	pbs_statfree(ss_save);
}

/**
 * @brief
 *	exit_qsub - issues the exit system call with the "exit" argument after
 * 	doing and needed library shutdown.
 *
 * @param[in] exitstatus integer value indiacting exit
 *
 * @return None
 *
 */
void
exit_qsub(int exitstatus)
{
	/* A thread that makes qsub exit, should try and acquire the Critical Section. */
	critical_section();

	if (cs_init == 1)
		/* Cleanup security library initializations before exiting */
		CS_close_app();

#ifdef BACKTRACE_SIZE
	if (exitstatus != 0) {
		int i, frames;
		void *bt_buf[BACKTRACE_SIZE];
		char **bt_strings;

		frames = backtrace(bt_buf, BACKTRACE_SIZE);
		printf("Backtrace has %d frames.\n", frames);
		bt_strings = backtrace_symbols(bt_buf, frames);
		if (bt_strings == NULL) {
			printf("No backtrace symbols present!\n");
		} else {
			for (i = 0; i < frames; i++) {
				printf("%s\n", bt_strings[i]);
			}
			free(bt_strings);
		}
	}
#endif

	exit(exitstatus);
}

/**
 * @brief
 *	strdup_esc_commas - duplicate a string escaping commas
 *	The string is duplicated with all commas in the original string
 *	escaped by preceding escape character.
 *
 * @param[in] str_to_dup - string to be duplicated
 *
 * @return
 * @retval string Succes
 * @retval NULL   Failure
 */
char *
strdup_esc_commas(char *str_to_dup)
{
	char *roaming = str_to_dup;
	char *endstr, *returnstr;

	if (str_to_dup == NULL)
		return NULL;

	returnstr = endstr = malloc(strlen(str_to_dup) * 2 + 2);
	/* even for an all-comma string, this should suffice */
	if (returnstr == NULL)
		return NULL; /* just return null on malloc failure */
	while (*roaming != '\0') {
		while (*roaming != '\0' && *roaming != ',')
			*(endstr++) = *(roaming++);
		if (*roaming == ',') {
			*(endstr++) = ESC_CHAR;
			*(endstr++) = ',';
			roaming++;
		}
	}
	*endstr = '\0';
	return (returnstr);
}

/**
 * @brief
 *	prints the usage format for qsub
 *
 */
static void
print_usage(void)
{
	static char usage2[] = "       qsub --version\n";
	extern char usage[];
	fprintf(stderr, "%s", usage);
	fprintf(stderr, "%s", usage2);
}

/* End of "Utility" functions. */

/* The following functions support the "Interactive Job" capability of PBS. */

/**
 * @brief
 * 	interactive_port - get a socket to listen to for "interactive" job
 *	When the "interactive" job is run, its standard in, out, and error
 *	will be connected to this socket.
 *
 * @return string
 * @retval portstring holding port info
 * @note exits from program on failure
 *
 */
static char *
interactive_port(void)
{
	pbs_socklen_t namelen;
	static char portstring[8];
	struct sockaddr_in myaddr;
	unsigned short port;

	if ((isatty(0) == 0) || (isatty(1) == 0)) {
		fprintf(stderr, "qsub:\tstandard input and output must be a terminal for\n"
				"\tinteractive job submission\n");
		exit_qsub(1);
	}
	comm_sock = socket(AF_INET, SOCK_STREAM, 0);
	if (comm_sock < 0) {
		perror("qsub: unable to obtain socket");
		exit_qsub(1);
	}
	myaddr.sin_family = AF_INET;
	myaddr.sin_addr.s_addr = INADDR_ANY;
	myaddr.sin_port = 0;
	if (bind(comm_sock, (struct sockaddr *) &myaddr, sizeof(myaddr)) < 0) {
		perror("qsub: unable to bind to socket");
		exit_qsub(1);
	}

	/* get port number assigned */

	namelen = sizeof(myaddr);
	if (getsockname(comm_sock, (struct sockaddr *) &myaddr, &namelen) < 0) {
		perror("qsub: unable to get port number");
		exit_qsub(1);
	}
	port = ntohs(myaddr.sin_port);
	(void) sprintf(portstring, "%u", (unsigned int) port);
	if (listen(comm_sock, 1) < 0) {
		perror("qsub: listen on interactive socket");
		exit_qsub(1);
	}

	return (portstring);
}

/**
 * @brief
 *	Shut and Close a socket
 *
 * @param	sock	file descriptor
 *
 * @return Void
 *
 */
static void
shut_close_sock(int sock)
{
	shutdown(sock, 2);
	closesocket(sock);
}

/**
 * @brief
 * 	send delete job request, disconnect with server and exit qsub
 *
 * @param[in]	ret	qsub exit code
 *
 * @return      void
 *
 */
void
bailout(int ret)
{
	int c;

	shut_close_sock(comm_sock);
	printf("Job %s is being deleted\n", new_jobname);
	c = cnt2server(server_out);
	if (c <= 0) {
		fprintf(stderr,
			"qsub: cannot connect to server %s (errno=%d)\n",
			pbs_server, pbs_errno);
		exit_qsub(1);
	}
	(void) pbs_deljob(c, new_jobname, NULL);
	pbs_disconnect(c);
	exit_qsub(ret);
}

/* The following functions support the "Block Job" capability of PBS. */

/**
 * @brief
 *	creates a socket and blocks the port
 *
 * @return char *
 * @retval portstring string holding port info
 *
 */
static char *
block_port(void)
{
	pbs_socklen_t namelen;
	static char portstring[8];
	struct sockaddr_in myaddr;
	unsigned short port;

	comm_sock = socket(AF_INET, SOCK_STREAM, 0);
	if (comm_sock < 0) {
		perror("qsub: unable to obtain socket");
		exit_qsub(1);
	}
	myaddr.sin_family = AF_INET;
	myaddr.sin_addr.s_addr = INADDR_ANY;
	myaddr.sin_port = 0;
	if (bind(comm_sock, (struct sockaddr *) &myaddr, sizeof(myaddr)) < 0) {
		perror("qsub: unable to bind to socket");
		exit_qsub(1);
	}

	/* get port number assigned */

	namelen = sizeof(myaddr);
	if (getsockname(comm_sock, (struct sockaddr *) &myaddr, &namelen) < 0) {
		perror("qsub: unable to get port number");
		exit_qsub(1);
	}
	port = ntohs(myaddr.sin_port);
	(void) sprintf(portstring, "%u", (unsigned int) port);
	DBPRT(("block_port: %s\n", portstring))

	if (listen(comm_sock, 1) < 0) {
		perror("qsub: listen on block socket");
		exit_qsub(1);
	}

	return (portstring);
}

int sig_happened = 0;

#define BAIL(message)             \
	if (ret != DIS_SUCCESS) { \
		fail = message;   \
		goto err;         \
	}

/**
 * @brief
 *	block - set up to wait for a job to end.
 *
 * @return Void
 * Exits on failre
 *
 */
static void
block(void)
{
	struct sockaddr_in from;
	pbs_socklen_t fromlen;
	char *jobid = "none";
	char *message = NULL;
	char *fail = NULL;
	int news;
	int ret;
	int version;
	int exitval;

#ifndef WIN32
	struct sigaction act;

	/* Catch SIGHUP, SIGINT, SIGQUIT and SIGTERM */

	sigemptyset(&act.sa_mask);
	act.sa_handler = blockint;
	act.sa_flags = 0;
	if ((sigaction(SIGHUP, &act, NULL) < 0) ||
	    (sigaction(SIGINT, &act, NULL) < 0) ||
	    (sigaction(SIGQUIT, &act, NULL) < 0) ||
	    (sigaction(SIGTERM, &act, NULL) < 0)) {
		perror("qsub: unable to catch signals");
		exit_qsub(1);
	}
#endif

retry:
	fromlen = sizeof(from);
	if ((news = accept(comm_sock, (struct sockaddr *) &from,
			   &fromlen)) < 0) {
#ifdef WIN32
		if (errno == WSAEINTR)
#else
		if (errno == EINTR)
#endif
		{
			fprintf(stderr, "qsub: wait for job %s "
					"interrupted by signal %d\n",
				new_jobname, sig_happened);
			bailout(2);
		}
		perror("qsub: accept error");
		exit_qsub(1);
	}
	DBPRT(("got connection from %s:%d\n", inet_ntoa(from.sin_addr), (int) ntohs(from.sin_port)))

	/*
	 * if SIGINT or SIGBREAK interrupt is raised, then child thread win_blockint()
	 * does job deletion and other related stuff. So main thread can exit now.
	 */

#ifdef WIN32
	if ((sig_happened == SIGINT) || (sig_happened == SIGBREAK))
		exit_qsub(3);
#endif

	/* When Mom connects back, the first thing that needs
	 * to happen is to engage in an authentication activity.
	 * Any return value other than CS_SUCCESS or CS_AUTH_USE_IFF
	 * means the authentication failed.
	 */
	ret = CS_client_auth(news);

	if ((ret != CS_SUCCESS) && (ret != CS_AUTH_USE_IFF)) {
		fprintf(stderr, "qsub: failed authentication with execution host\n");
		shut_close_sock(news);
		goto retry;
	}

	DIS_tcp_funcs();
	version = disrsi(news, &ret);
	if (ret != DIS_SUCCESS) {
		/*
		 * We couldn't read data so try again if it is a port scan.
		 */
		shut_close_sock(news);
		goto retry;
	}
	if (version != 1) {
		fprintf(stderr, "qsub: unknown protocol version %d\n", version);
		shut_close_sock(news);
		goto retry;
	}

	jobid = disrst(news, &ret);
	if ((ret != DIS_SUCCESS) || (strcmp(jobid, new_jobname) != 0)) {
		fprintf(stderr, "qsub: Unknown Job Identifier %s\n", jobid);
		shut_close_sock(news);
		goto retry;
	}

	/* after getting the correct jobid, give up on error */
	message = disrst(news, &ret);
	BAIL("message")
	if (message != NULL && *message != '\0') { /* non-null message */
		fprintf(stderr, "qsub: %s %s\n", jobid, message);
		exit_qsub(3);
	}
	exitval = disrsi(news, &ret);
	BAIL("exitval");
	exit_qsub(exitval);

err:
	fprintf(stderr, "qsub: Bad Request Protocol, %s\n",
		((fail != NULL) && (*fail != '\0')) ? fail : "unknown error");
	exit_qsub(3);
}

/* End of "Block Job" functions. */

/* End of "Authentication" functions. */

/*
 * The following functions support the "Options Processing"
 * functionality of qsub.
 */

/**
 * @brief
 *  This function processes all the options specified while submitting a job. It
 *  validates all these options and sets their corresponding flags.
 *
 * @param[in] argc  Number of options present in argv.
 * @param[in] argv  An array containing all the options and their values.
 * @param[in] passet The value that will be used to set the options. It can have
 *                   value as CMDLINE (for command line options), CMDLINE-1 (for
 *                   job script options), CMDLINE-2 (server default options).
 *
 * @return int - It returns number of erroneous options processed.
 *
 */
static int
process_opts(int argc, char **argv, int passet)
{
	int i;
	int c;
	char *erp;
	int errflg = 0;
	time_t after;
	char a_value[512];
	char *keyword;
	char *valuewd;
	char *pc;
	struct attrl *pattr = NULL;
	size_t N_len = 0;
	int ddash_index = -1;

	extern char GETOPT_ARGS[];

/*
 * The following macro, together the value of passet is used
 * to enforce the following rules:
 * 1. option on the command line take precedence over those in script directives.
 * 2. With in the command line or within the script, the last occurance of an option takes
 *    precedence over the earlier occurance.
 */

/*
 * The passet value is saved in the opt register. The option will
 * only be set if the value of passet is greater then or equal to the
 * opt regiester.
 */
#define if_cmd_line(x) if (x <= passet)

	if (passet != CMDLINE) {
#if defined(linux) || defined(WIN32)
		optind = 0; /* prime getopt's starting point */
#else
		optind = 1; /* prime getopt's starting point */
#endif
	}
	while ((c = getopt(argc, argv, GETOPT_ARGS)) != EOF) {
		/*
		 * qsub uses "--" to specify the executable to run for a job,
		 * so, if "--" is used as a value, we need to make sure that
		 * there is another "--" for providing the executable name (if any).
		 */
		if (optarg && (strcmp(optarg, "--") == 0))
			ddash_index = optind - 1;

		switch (c) {
			case 'a':
				if_cmd_line(a_opt)
				{
					a_opt = passet;
					if ((after = cvtdate(optarg)) < 0) {
						fprintf(stderr, "qsub: illegal -a value\n");
						errflg++;
						break;
					}
					sprintf(a_value, "%ld", (long) after);
					(void) set_attr_error_exit(&attrib, ATTR_a, a_value);
				}
				break;
			case 'A':
				if_cmd_line(A_opt)
				{
					A_opt = passet;
					(void) set_attr_error_exit(&attrib, ATTR_A, optarg);
				}
				break;
			case 'P':
				if_cmd_line(P_opt)
				{
					P_opt = passet;
					(void) set_attr_error_exit(&attrib, ATTR_project, optarg);
				}
				break;
			case 'c':
				if_cmd_line(c_opt)
				{
					c_opt = passet;
					while (isspace((int) *optarg))
						optarg++;
					pc = optarg;
					if (strlen(optarg) == 1) {
						if (*pc == 'u') {
							fprintf(stderr, "qsub: illegal -c value\n");
							errflg++;
							break;
						}
					}
					set_attr_error_exit(&attrib, ATTR_c, optarg);
				}
				break;
			case 'C':
				if_cmd_line(C_opt)
				{
					C_opt = passet;
					snprintf(dir_prefix, sizeof(dir_prefix), "%s", optarg);
				}
				break;
			case 'e':
				if_cmd_line(e_opt)
				{
					e_opt = passet;
					set_attr_error_exit(&attrib, ATTR_e, optarg);
				}
				break;
			case 'h':
				if_cmd_line(h_opt)
				{
					h_opt = passet;
					set_attr_error_exit(&attrib, ATTR_h, "u");
				}
				break;
			case 'f':
				no_background = 1;
				break;
#if !defined(PBS_NO_POSIX_VIOLATION)
			case 'I':
				if (J_opt != 0) {
					fprintf(stderr, "%s", INTER_ARRAY);
					errflg++;
					break;
				}
				if_cmd_line(Interact_opt)
				{
					Interact_opt = passet;
					if (block_opt != FALSE) {
						fprintf(stderr, "%s", INTER_BLOCK_WARN);
						block_opt = FALSE;
					}
					if (roptarg_inter == TRUE) {
						fprintf(stderr, "%s", INTER_RERUN_WARN);
					}
					set_attr_error_exit(&attrib, ATTR_inter, interactive_port());
				}
				break;
#endif /* PBS_NO_POSIX_VIOLATION */
			case 'j':
				if_cmd_line(j_opt)
				{
					j_opt = passet;
					set_attr_error_exit(&attrib, ATTR_j, optarg);
				}
				break;
			case 'J':
				if (Interact_opt != FALSE) {
					fprintf(stderr, "%s", INTER_ARRAY);
					errflg++;
					break;
				}
				if (roptarg != 'y') {
					fprintf(stderr, "%s", NO_RERUN_ARRAY);
					errflg++;
					break;
				}
				if_cmd_line(J_opt)
				{
					char *p;
					J_opt = passet;
					p = strpbrk(optarg, "%");
					if (p != NULL)
						*p = '\0';
					set_attr_error_exit(&attrib, ATTR_J, optarg);
					if (p != NULL) {
						if (max_run_opt == FALSE) {
							max_run_opt = TRUE;
							set_attr_error_exit(&attrib, ATTR_max_run_subjobs, ++p);
						} else {
							fprintf(stderr, "%s", MULTIPLE_MAX_RUN);
							errflg++;
							break;
						}
					}
				}
				break;
			case 'k':
				if_cmd_line(k_opt)
				{
					k_opt = passet;
					set_attr_error_exit(&attrib, ATTR_k, optarg);
				}
				break;
			case 'l':
				l_opt = passet;
				if ((i = set_resources(&attrib, optarg, (passet == CMDLINE), &erp))) {
					if (i > 1) {
						pbs_prt_parse_err("qsub: illegal -l value\n", optarg,
								  (int) (erp - optarg), i);
					} else
						fprintf(stderr, "qsub: illegal -l value\n");
					errflg++;
				}
				break;
			case 'm':
				if_cmd_line(m_opt)
				{
					m_opt = passet;
					while (isspace((int) *optarg))
						optarg++;
					set_attr_error_exit(&attrib, ATTR_m, optarg);
				}
				break;
			case 'M':
				if_cmd_line(M_opt)
				{
					M_opt = passet;
					set_attr_error_exit(&attrib, ATTR_M, optarg);
				}
				break;
			case 'N':
				if_cmd_line(N_opt)
				{
					N_opt = passet;
					/* If ATTR_N is not set previously */
					if (get_attr(attrib, ATTR_N, NULL) == NULL) {
						set_attr_error_exit(&attrib, ATTR_N, optarg);
					}
					/* If N_opt is not set previously but if ATTR_N is set
					 * earlier directly without verification based on the
					 * job script name and if there is a value for ATTR_N
					 * after parsing the job script for PBS directives
					 * replace the earlier value with the current value
					 * for this attribute
					 */
					else {
						for (pattr = attrib; pattr; pattr = pattr->next) {
							if (strcmp(pattr->name, ATTR_N) == 0) {
								N_len = strlen(optarg);
								if (strlen(pattr->value) < N_len) {
									pattr->value = (char *) realloc(pattr->value, N_len + 1);
									if (pattr->value == NULL) {
										fprintf(stderr, "Out of memory\n");
										exit(2);
									}
								}
								strcpy(pattr->value, optarg); /* safe because we just allocated enough space */
							}
						}
					}
				}
				break;
			case 'o':
				if_cmd_line(o_opt)
				{
					o_opt = passet;
					set_attr_error_exit(&attrib, ATTR_o, optarg);
				}
				break;
			case 'p':
				if_cmd_line(p_opt)
				{
					p_opt = passet;
					while (isspace((int) *optarg))
						optarg++;
					set_attr_error_exit(&attrib, ATTR_p, optarg);
				}
				break;
			case 'q':
				if_cmd_line(q_opt)
				{
					q_opt = passet;
					snprintf(destination, sizeof(destination), "%s", optarg);
				}
				break;
			case 'r':
				if_cmd_line(r_opt)
				{
					r_opt = passet;
					if (strlen(optarg) != 1) {
						fprintf(stderr, "qsub: illegal -r value\n");
						errflg++;
						break;
					}
					if (*optarg != 'y' && *optarg != 'n') {
						fprintf(stderr, "qsub: illegal -r value\n");
						errflg++;
						break;
					} else if ((*optarg == 'n') && (J_opt != 0)) {
						fprintf(stderr, "%s", NO_RERUN_ARRAY);
						errflg++;
						break;
					}
					if (*optarg == 'y') {
						roptarg_inter = TRUE;
						if (Interact_opt)
							fprintf(stderr, "%s", INTER_RERUN_WARN);
					}
					roptarg = *optarg;
					set_attr_error_exit(&attrib, ATTR_r, optarg);
				}
				break;
			case 'R':
				if_cmd_line(R_opt)
				{
					R_opt = passet;
					set_attr_error_exit(&attrib, ATTR_R, optarg);
				}
				break;
			case 'S':
				if_cmd_line(S_opt)
				{
					S_opt = passet;
					set_attr_error_exit(&attrib, ATTR_S, optarg);
				}
				break;
			case 'u':
				if_cmd_line(u_opt)
				{
					u_opt = passet;
					set_attr_error_exit(&attrib, ATTR_u, optarg);
				}
				break;
			case 'v':
				if_cmd_line(v_opt)
				{
					v_opt = passet;
					free(v_value);
					/*
					 * Need to change '\' to '/' before expanding the
					 * environment because '\' is used to protect commas
					 * inside quoted values.
					 */
					fix_path(optarg, 1);
					v_value = expand_varlist(optarg);
					if (v_value == NULL)
						exit(1);
				}
				break;
			case 'V':
				if_cmd_line(V_opt)
				{
					V_opt = passet;
				}
				break;
			case 'W':
				while (isspace((int) *optarg))
					optarg++;
				if (strlen(optarg) == 0) {
					fprintf(stderr, "%s", BAD_W);
					errflg++;
					break;
				}
				fix_path(optarg, 2);
				i = parse_equal_string(optarg, &keyword, &valuewd);

				/*
				 * All the arguments to option 'W' are
				 * accepted in the format of -Wattrname=value.
				 */

				while (i == 1) {
					if (strcmp(keyword, ATTR_depend) == 0) {
						if_cmd_line(Depend_opt)
						{
							Depend_opt = passet;
							set_attr_error_exit(&attrib, ATTR_depend, valuewd);
						}
					} else if (strcmp(keyword, ATTR_stagein) == 0) {
						if_cmd_line(Stagein_opt)
						{
							Stagein_opt = passet;
							set_attr_error_exit(&attrib, ATTR_stagein, valuewd);
						}
					} else if (strcmp(keyword, ATTR_stageout) == 0) {
						if_cmd_line(Stageout_opt)
						{
							Stageout_opt = passet;
							set_attr_error_exit(&attrib, ATTR_stageout, valuewd);
						}
					} else if (strcmp(keyword, ATTR_sandbox) == 0) {
						if_cmd_line(Sandbox_opt)
						{
							Sandbox_opt = passet;
							set_attr_error_exit(&attrib, ATTR_sandbox, valuewd);
						}
					} else if (strcmp(keyword, ATTR_g) == 0) {
						if_cmd_line(Grouplist_opt)
						{
							Grouplist_opt = passet;
							set_attr_error_exit(&attrib, ATTR_g, valuewd);
						}
					} else if (strcmp(keyword, ATTR_inter) == 0) {
						if_cmd_line(Interact_opt)
						{
							if (J_opt != 0) {
								fprintf(stderr, "%s", INTER_ARRAY);
								errflg++;
								break;
							}
							/*
							 * SPID 232472: can't set interactive attribute to false
							 * Problem: "qsub -W interactive=false" throws an error
							 * Cause: There should be check to compare the user value
							 *   with "false" string and accordingly decide whether it
							 *   is an interactive job or not.
							 * Solution: Added additional checks which will not set
							 *   Interact_opt and will not call set_attr_error_exit() to create
							 *   interactive port if user gives a value "false"
							 */
							if (!(strcasecmp(valuewd, "true"))) {
								Interact_opt = passet;
								set_attr_error_exit(&attrib, ATTR_inter, interactive_port());
							} else if (!(strcasecmp(valuewd, "false"))) {
								/* Do Nothing, let it run as a non-interactive job */
							} else {
								/* Any value other than true/false is not acceptable */
								fprintf(stderr, "%s", BAD_W);
								errflg++;
								break;
							}
							if (roptarg_inter == TRUE) {
								fprintf(stderr, "%s", INTER_RERUN_WARN);
							}
							/* check if both block and interactive are true */
							if ((block_opt != FALSE) && (Interact_opt)) {
								fprintf(stderr, "%s", INTER_BLOCK_WARN);
								block_opt = FALSE;
								break;
							}
						}
					} else if (strcmp(keyword, ATTR_block) == 0) {
						if_cmd_line(block_opt)
						{
							if (!(strcasecmp(valuewd, "true"))) {
								block_opt = passet;
							} else if (!(strcasecmp(valuewd, "false"))) {
								/* Do Nothing, Let it run as a non-blocking job */
							} else {
								/* Any value other than true/false is not acceptable */
								fprintf(stderr, "%s", BAD_W);
								errflg++;
								break;
							}
							if ((Interact_opt != FALSE) && (block_opt == passet)) {
								fprintf(stderr, "%s", INTER_BLOCK_WARN);
								block_opt = FALSE;
								break;
							}
						}
					} else if (strcmp(keyword, ATTR_resv_start) == 0) {
						if_cmd_line(Resvstart_opt)
						{
							Resvstart_opt = passet;
							if ((after = cvtdate(valuewd)) < 0) {
								fprintf(stderr, "%s", BAD_W);
								errflg++;
								break;
							}
							sprintf(a_value, "%ld", (long) after);
							set_attr_error_exit(&attrib, ATTR_resv_start, a_value);
						}
					} else if (strcmp(keyword, ATTR_resv_end) == 0) {
						if_cmd_line(Resvend_opt)
						{
							Resvend_opt = passet;
							if ((after = cvtdate(valuewd)) < 0) {
								fprintf(stderr, "%s", BAD_W);
								errflg++;
								break;
							}
							sprintf(a_value, "%ld", (long) after);
							set_attr_error_exit(&attrib, ATTR_resv_end, a_value);
						}
					} else if (strcmp(keyword, ATTR_cred) == 0) {
						if_cmd_line(cred_opt)
						{
							cred_opt = passet;
							snprintf(cred_name, sizeof(cred_name), "%s", valuewd);
							set_attr_error_exit(&attrib, ATTR_cred, valuewd);
						}
					} else if (strcmp(keyword, ATTR_tolerate_node_failures) == 0) {
						if_cmd_line(tolerate_node_failures_opt)
						{
							tolerate_node_failures_opt = passet;
							set_attr_error_exit(&attrib, ATTR_tolerate_node_failures, valuewd);
						}
					} else if (strcmp(keyword, ATTR_max_run_subjobs) == 0) {
						if (max_run_opt == FALSE) {
							max_run_opt = TRUE;
							set_attr_error_exit(&attrib, keyword, valuewd);
						} else {
							fprintf(stderr, "%s", MULTIPLE_MAX_RUN);
							errflg++;
							break;
						}
					} else {
						set_attr_error_exit(&attrib, keyword, valuewd);
					}
					i = parse_equal_string(NULL, &keyword, &valuewd);
				} /* bottom of long while loop */
				if (i == -1) {
					fprintf(stderr, "%s", BAD_W);
					errflg++;
				}
				break;

			case 'X':
				if_cmd_line(Forwardx11_opt)
				{
					Forwardx11_opt = passet;
#if !defined(PBS_NO_POSIX_VIOLATION) && !defined(WIN32)
					if (!(display = getenv("DISPLAY"))) {
						fprintf(stderr, "qsub: DISPLAY not set\n");
						errflg++;
					}
#endif
				}
				break;
			case 'G':
				if_cmd_line(gui_opt)
				{
					gui_opt = passet;
					set_attr_error_exit(&attrib, ATTR_GUI, "TRUE");
				}
				break;
			case 'z':
				if_cmd_line(z_opt) z_opt = passet;
				break;
			case '?':
			default:
				errflg++;
		}
	}
	if ((block_opt == passet) && (Interact_opt == FALSE))
		set_attr_error_exit(&attrib, ATTR_block, block_port());
	if ((Forwardx11_opt == CMDLINE) && (Interact_opt == FALSE) && (errflg == 0)) {
		fprintf(stderr, "qsub: X11 Forwarding possible only for "
				"interactive jobs\n");
		exit_qsub(1);
	}
	if ((gui_opt == CMDLINE) && (Interact_opt == FALSE)) {
		fprintf(stderr, INTER_GUI_WARN);
		gui_opt = FALSE;
		exit_qsub(1);
	}

	if (errflg == 0 && J_opt == 0 && get_attr(attrib, ATTR_m, NULL) != NULL &&
	    strchr(get_attr(attrib, ATTR_m, NULL), 'j') != NULL) {
		fprintf(stderr, "qsub: mail option 'j' can not be used without array job\n");
		exit_qsub(1);
	}

	/*
	 * If argv[optind] points to '--' string, then
	 * decrement optind, so that it would always point
	 * to first non-command line option.
	 * And also confirm if "--" was consumed by getopt
	 * and not used as an argument value.
	 * If used as an argument value, we cannot use it as
	 * an indicator that an executable name follows the "--".
	 */
	if (strcmp(argv[optind - 1], "--") == 0) {
		if (ddash_index != optind - 1)
			optind--;
		else
			errflg++;
	}

	if ((optind != 0) && (argc > 1) && (argv[optind] != NULL)) {
		/* Now, optind is pointing to first non-command line option */
		char *s = argv[optind];
		if ((s[0] == '-') && (s[1] == '-') && (s[2] == '\0')) {
			/* optind points to '--', it should not be last character */
			if (optind == (argc - 1))
				errflg++;
		} else {
			/* optind points to 'script-file path' */
			/* It should be a last argument in command-line options */
			if (optind != (argc - 1))
				errflg++;
		}
	}
	if (!errflg && passet != CMDLINE) {
		errflg = (optind != argc);
	}
	/* use PBS_SHELL if specified only if -S was not specified */
	if (S_opt == FALSE) {
		char *c = getenv("PBS_SHELL");
		if (c)
			set_attr_error_exit(&attrib, ATTR_S, c);
	}

	if (u_opt && cred_name[0]) {
		fprintf(stderr, "qsub: credential incompatable with -u\n");
		errflg++;
	}
	return (errflg);
}

/**
 * @brief
 *  Process special arguments.
 *  The "--" argument indicates an executable and possible arguments to that
 *  exectuable. qsub will treat that executable and its arguments as the job
 *  rather than reading from a job script.
 *
 * @param[in]  argc         - argument count
 * @param[in]  argv         - pointer to array of argument variables
 * @param[out] script       - path of job script
 * @return     command_flag - indicates whether an executable was specified instead of a job script
 */
static int
process_special_args(int argc, char **argv, char *script)
{
	int command_flag = 0;
	char *arg_list = NULL;
	if (optind < argc) {
		if (strcmp(argv[optind], "--") == 0) {
			command_flag = 1;
			/* set executable */
			set_attr_error_exit(&attrib, ATTR_executable, argv[optind + 1]);
			if (argc > (optind + 2)) {
				/* user has specified arguments to executable as well. */
				arg_list = encode_xml_arg_list(optind + 2, argc, argv);
				if (arg_list == NULL) {
					fprintf(stderr, "qsub: out of memory\n");
					exit_qsub(2);
				} else {
					/* set argument list */
					set_attr_error_exit(&attrib, ATTR_Arglist, arg_list);
					free(arg_list);
					arg_list = NULL;
				}
			}
			if (!N_opt) /* '-N' is not set */
				set_attr_error_exit(&attrib, ATTR_N, "STDIN");
		} else {
			if (optind + 1 != argc) {
				/* argument is a job script, it should be last */
				print_usage();
				exit_qsub(2);
			}
			snprintf(script, MAXPATHLEN, "%s", argv[optind]);
		}
	}
	return command_flag;
}

/**
 * @brief
 * 	processes and creates arguments passed for qsub
 *
 * @param[in] argc - argument count
 * @param[in] argv - pointer to array of argument variables
 * @param[in] line - character pointer for whole line
 *
 */
static void
make_argv(int *argc, char *argv[], char *line)
{
	char *l, *b, *c;
	char static_buffer[MAX_LINE_LEN + 1];
	char *buffer;
	int line_len = 0;
	int len;
	char quote;
	int i;

	*argc = 0;
	argv[(*argc)++] = "qsub";
	l = line;
	line_len = strlen(line);
	if (line_len > MAX_LINE_LEN) {
		buffer = malloc(line_len + 1);
		if (buffer == NULL) {
			fprintf(stderr, "qsub: out of memory\n");
			exit_qsub(2);
		}
	} else
		buffer = static_buffer;
	b = buffer;
	while (isspace(*l))
		l++;
	c = l;
	while (*c != '\0') {
		if ((*c == '"') || (*c == '\'')) {
			quote = *c;
			c++;
			while ((*c != quote) && *c)
				*b++ = *c++;
			if (*c == '\0') {
				fprintf(stderr, "qsub: unmatched %c\n", *c);
				exit_qsub(1);
			}
			c++;
		} else if (*c == ESC_CHAR) {
			c++;
			*b++ = *c++;
		} else if (isspace(*c)) {
			len = c - l;
			free(argv[*argc]);
			argv[*argc] = (char *) malloc(len + 1);
			if (argv[*argc] == NULL) {
				fprintf(stderr, "qsub: out of memory\n");
				exit_qsub(2);
			}
			*b = '\0';
			strcpy(argv[(*argc)++], buffer);
			while (isspace(*c))
				c++;
			l = c;
			b = buffer;
		} else
			*b++ = *c++;
	}
	if (c != l) {
		len = c - l;
		free(argv[*argc]);
		argv[*argc] = (char *) malloc(len + 1);
		if (argv[*argc] == NULL) {
			fprintf(stderr, "qsub: out of memory\n");
			exit_qsub(2);
		}
		*b = '\0';
		strcpy(argv[(*argc)++], buffer);
	}
	i = *argc;
	/*
	 * free and null any pointers used for the prior call that are not used
	 * for this line. Otherwise the argv array would not be null terminated
	 */
	while (argv[i] != NULL) {
		free(argv[i]);
		argv[i++] = NULL;
	}
	if (buffer != static_buffer)
		free(buffer);
}

/**
 * @brief
 *      Create and process qsub argument list from the string 'opts'
 *
 * @param[in]	opts     - The qsub options as single parameter.
 * @param[in]   opt_pass - priority set based on precedence.
 *
 * @return      int
 * @retval	>0 - Failure - Other than PBS directive error.
 * @retval      -1 - Failure - PBS directive error.
 * @retval	 0 - Success
 *
 */
int
do_dir(char *opts, int opt_pass, char *retmsg, size_t ret_size)
{
	int argc;
	int ret = -1;
	int index = 0;
	int len = 0;
	int nxt_pos = 0;
	size_t max_size = ret_size - 2 /* 2 deducted for adding newline at end */;
#define MAX_ARGV_LEN 128
	static char *vect[MAX_ARGV_LEN + 1];

	make_argv(&argc, vect, opts);
	ret = process_opts(argc, vect, opt_pass);
	if ((ret != 0) && (opt_pass != CMDLINE)) {
		nxt_pos = snprintf(retmsg, max_size, "qsub: directive error: ");
		if (nxt_pos < 0)
			return (ret);
		max_size = max_size - nxt_pos;
		for (index = 1; index < argc; index++) {
			/* +1 is added to strlen(vect[index]) to reserve space */
			if ((max_size > 0) && (max_size > strlen(vect[index]) + 1)) {
				len = snprintf(retmsg + nxt_pos, max_size, "%s ", vect[index]);
				if (len < 0)
					break;
				nxt_pos = nxt_pos + len;
				max_size = max_size - len;
			} else {
				break;
			}
		}
		snprintf(retmsg + nxt_pos, 2, "\n");
		return (-1);
	}
	return (ret);
}

/*
 * @brief
 *	set_opt_defaults - if not already set, set certain job attributes to
 *	their default value
 *
 */
static void
set_opt_defaults(void)
{
	if (c_opt == FALSE)
		set_attr_error_exit(&attrib, ATTR_c, CHECKPOINT_UNSPECIFIED);
	if (h_opt == FALSE)
		set_attr_error_exit(&attrib, ATTR_h, NO_HOLD);
	if (j_opt == FALSE)
		set_attr_error_exit(&attrib, ATTR_j, NO_JOIN);
	if (k_opt == FALSE)
		set_attr_error_exit(&attrib, ATTR_k, NO_KEEP);
	if (m_opt == FALSE)
		set_attr_error_exit(&attrib, ATTR_m, MAIL_AT_ABORT);
	if (p_opt == FALSE)
		set_attr_error_exit(&attrib, ATTR_p, "0");
	if (r_opt == FALSE)
		set_attr_error_exit(&attrib, ATTR_r, "TRUE");
}

/* End of "Options Processing" functions. */

/**
 * @brief
 *	Returns the directory prefix string, which is chosen from the following possibilities:
 *	1. the prefix parameter, if not empty
 *	2. an empty string
 *	3. the PBS_DPREFIX environment variable
 *	4. the PBS_DPREFIX_DEFAULT constant
 *
 * @param[in] prefix - string to be prefixed
 * @param[in] diropt - boolean value indicating directory prefix to be set or not
 *
 * @return String
 * @retval Success - pbs directory prefix
 * @retval Failure - NULL
 *
 */
static char *
set_dir_prefix(char *prefix, int diropt)
{
	char *s;

	if (notNULL(prefix))
		return (prefix);
	else if (diropt != FALSE)
		return ("");
	else if ((s = getenv("PBS_DPREFIX")) != NULL)
		return (s);
	else
		return (PBS_DPREFIX_DEFAULT);
}

/**
 * @brief
 * Read the job script from a file or stdin.
 *
 * @param[in] script - path of job script to read from
 */
static void
read_job_script(char *script)
{
	extern char read_script_msg[];
	int errflg; /* error code from get_script() */
	struct stat statbuf;
	char *bnp;
	char basename[MAXPATHLEN + 1]; /* base name of script for job name*/
	FILE *f;		       /* FILE pointer to the script */

	/* if script is empty, get standard input */
	if ((strcmp(script, "") == 0) || (strcmp(script, "-") == 0)) {
		/* if this is a terminal, print a short info */
		if (isatty(STDIN_FILENO) && Interact_opt == FALSE) {
			printf("%s", read_script_msg);
		}

		if (!N_opt)
			set_attr_error_exit(&attrib, ATTR_N, "STDIN");
		if (Interact_opt == FALSE) {
			errflg = get_script(stdin, script_tmp, set_dir_prefix(dir_prefix, C_opt));
			if (errflg > 0) {
				(void) unlink(script_tmp);
				exit_qsub(1);
			} else if (errflg < 0) {
				exit_qsub(1);
			}
		}
	} else { /* non-empty script, read it for directives */
		if (stat(script, &statbuf) < 0) {
			perror("qsub: script file:");
			exit_qsub(1);
		}
		if (!S_ISREG(statbuf.st_mode)) {
			fprintf(stderr, "qsub: script not a file\n");
			exit_qsub(1);
		}
		if ((f = fopen(script, "r")) != NULL) {
			if (!N_opt) {
				if ((bnp = strrchr(script, (int) '/')) != NULL)
					bnp++;
				else
					bnp = script;

				snprintf(basename, sizeof(basename), "%s", bnp);
				/*
				 * set ATTR_N directly - verification would be done
				 * by IFL later
				 */
				set_attr_error_exit(&attrib, ATTR_N, basename);
			}
			errflg = get_script(f, script_tmp, set_dir_prefix(dir_prefix, C_opt));
			if (errflg > 0) {
				(void) unlink(script_tmp);
				exit_qsub(1);
			} else if (errflg < 0) {
				exit_qsub(1);
			}
			(void) fclose(f);
			f = NULL;
		} else {
			perror("qsub: opening script file:");
			exit_qsub(8);
		}
	}
}

/* End of "Job Script" functions. */

/* The following functions supports the "Environment Variables" feature of qsub. */

/**
 * @brief
 *	Constructs the basic comma-separated environment variables
 *	list string for a PBS job.
 *
 * @return	char *
 * @retval	NULL for failure.
 * @retval	A comma-separated list of environment variable=value entries.
 *
 */
static char *
job_env_basic(void)
{
	char *job_env = NULL;
	char *s = NULL;
	char *c = NULL;
	char *p = NULL;
	char *env = NULL;
#ifdef WIN32
	OSVERSIONINFO os_info;
#else
	struct utsname uns;
#endif
	int len = 0;
	char *getcwd();

	/* Calculate how big to make the variable string. */
	len = 0;
	env = strdup_esc_commas(getenv("HOME"));
	if (env != NULL) {
		len += strlen(env);
		free(env);
	}
	env = strdup_esc_commas(getenv("LANG"));
	if (env != NULL) {
		len += strlen(env);
		free(env);
	}
	env = strdup_esc_commas(getenv("LOGNAME"));
	if (env != NULL) {
		len += strlen(env);
		free(env);
	}
	env = strdup_esc_commas(getenv("PATH"));
	if (env != NULL) {
		len += strlen(env);
		free(env);
	}
	env = strdup_esc_commas(getenv("MAIL"));
	if (env != NULL) {
		len += strlen(env);
		free(env);
	}
	env = strdup_esc_commas(getenv("SHELL"));
	if (env != NULL) {
		len += strlen(env);
		free(env);
	}
	env = strdup_esc_commas(getenv("TZ"));
	if (env != NULL) {
		len += strlen(env);
		free(env);
	}
	len += PBS_MAXHOSTNAME;
	len += MAXPATHLEN;
	len *= 2; /* Double it for all the commas, etc. */

	if ((job_env = (char *) malloc(len)) == NULL) {
		fprintf(stderr, "malloc failure (errno %d)\n", errno);
		return NULL;
	}
	memset(job_env, '\0', len);

	/* Send the required variables with the job. */
	c = strdup_esc_commas(getenv("HOME"));
	fix_path(c, 1);
	strcat(job_env, "PBS_O_HOME=");
	if (c != NULL) {
		strcat(job_env, c);
		free(c);
	} else
		strcat(job_env, "/");
	c = strdup_esc_commas(getenv("LANG"));
	if (c != NULL) {
		strcat(job_env, ",PBS_O_LANG=");
		strcat(job_env, c);
		free(c);
	}
	c = strdup_esc_commas(getenv("LOGNAME"));
	if (c != NULL) {
		strcat(job_env, ",PBS_O_LOGNAME=");
		strcat(job_env, c);
		free(c);
	}
	c = strdup_esc_commas(getenv("PATH"));
	fix_path(c, 1);
	if (c != NULL) {
		strcat(job_env, ",PBS_O_PATH=");
		strcat(job_env, c);
		free(c);
	}
	c = strdup_esc_commas(getenv("MAIL"));
	fix_path(c, 1);
	if (c != NULL) {
		strcat(job_env, ",PBS_O_MAIL=");
		strcat(job_env, c);
		free(c);
	}
	c = strdup_esc_commas(getenv("SHELL"));
	fix_path(c, 1);
	if (c != NULL) {
		strcat(job_env, ",PBS_O_SHELL=");
		strcat(job_env, c);
		free(c);
	}

	c = strdup_esc_commas(getenv("TZ"));
	if (c != NULL) {
		strcat(job_env, ",PBS_O_TZ=");
		strcat(job_env, c);
		free(c);
	}

	/*
	 * Don't detect the hostname here because it utilizes network services
	 * that slow everthing down. PBS_O_HOST is set in the daemon later on.
	 */

	/* get current working directory, use $PWD if available, it is more
	 * NFS automounter "friendly". But must double check that is right
	 */
	s = job_env + strlen(job_env);
	strcat(job_env, ",PBS_O_WORKDIR=");
	c = getenv("PWD");
	if (c != NULL) {
		struct stat statbuf;
		dev_t dev;
		ino_t ino;

		if (stat(c, &statbuf) < 0) {
			/* cannot stat, cannot trust it */
			c = NULL;
		} else {
			dev = statbuf.st_dev;
			ino = statbuf.st_ino;
			if (stat(".", &statbuf) < 0) {
				perror("qsub: cannot stat current directory: ");
				free(job_env);
				return NULL;
			}
			/* compare against "." */
			if ((dev != statbuf.st_dev) || (ino != statbuf.st_ino))
				/* "." and $PWD is different, cannot trust it */
				c = NULL;
		}
	}

	if (c == NULL) {
		p = c = job_env + strlen(job_env);
		if (getcwd(c, MAXPATHLEN) == NULL)
			c = NULL;
	} else
		p = job_env + strlen(job_env);

	if (c != NULL) {
		char *c_escaped = NULL;

		/* save current working dir for daemon */
		snprintf(qsub_cwd, sizeof(qsub_cwd), "%s", c);
		/* get UNC path (if available) if it is mapped drive */
		get_uncpath(c);
		c_escaped = strdup_esc_commas(c);
		if (c_escaped != NULL) {
			fix_path(c_escaped, 1);
			pbs_strncpy(p, c_escaped, len - (p - job_env));
			free(c_escaped);
			c_escaped = NULL;
		} else
			*s = '\0';
	} else
		*s = '\0';

#ifdef WIN32 /* Windows */
	os_info.dwOSVersionInfoSize = sizeof(OSVERSIONINFO);
	if (GetVersionEx(&os_info)) {
		switch (os_info.dwPlatformId) {
			case 0:
				strcat(job_env, ",PBS_O_SYSTEM=VER_PLATFORM_WIN32s");
				break;
			case 1:
				strcat(job_env, ",PBS_O_SYSTEM=VER_PLATFORM_WIN32_WINDOWS");
				break;
			case 2:
				strcat(job_env, ",PBS_O_SYSTEM=VER_PLATFORM_WIN32_NT");
				break;
		}
	}
#else /* Unix */
	if (uname(&uns) != -1) {
		strcat(job_env, ",PBS_O_SYSTEM=");
		strcat(job_env, uns.sysname);
	}
#endif
	else {
		perror("qsub: cannot get uname info:");
		free(job_env);
		return NULL;
	}

	return (job_env);
}

/**
 * @brief
 *	Converts an array of environment variable=value strings,
 *	into a comma-separated variables list string that can be
 *	exported to a job.
 *
 * @par	 NOTE: Variables in the list beginning with "PBS_O" are ignored
 *	 as these will be preconstructed somewhere else.
 *
 * @param[in]	envp - aray of strings making up the current environment.
 *
 * @return      char *
 * @retval      NULL - Failure
 * @retval      A comma-separated list of environment variables and values.
 *		The returned string is malloc-ed so it must be freed later.
 */
static char *
env_array_to_varlist(char **envp)
{
	char **evp;
	int len;
	char *job_env = NULL;
	char *s;

	if (envp == NULL) {
		fprintf(stderr, "env_array_to_varlist: no envp array!\n");
		return NULL;
	}

	evp = envp;
	len = 0;
	while (notNULL(*evp)) {
		len += strlen(*evp);
		evp++;
	}
	len += len; /* Double it for all the commas, etc. */

	if ((len > 0) && ((job_env = (char *) malloc(len)) == NULL)) {
			fprintf(stderr, "env_array_to_varlist: malloc failure errno=%d", errno);
			return NULL;
	}

	*job_env = '\0';

	evp = envp;
	while (notNULL(*evp)) {
		s = *evp;
		while ((*s != '=') && *s)
			++s;
		*s = '\0';
		if (strncmp(*evp, PBS_O_ENV, sizeof(PBS_O_ENV) - 1) != 0) {
			/* do not add PBS_O_* env variables, as these are set by qsub */
			strcat(job_env, ",");
			strcat(job_env, *evp);
			strcat(job_env, "=");
			fix_path(s + 1, 1);
			(void) copy_env_value(job_env, s + 1, 1);
		}
		*s = '=';
		evp++;
	}

	return (job_env);
}

/**
 * @brief
 *	Adds to the global 'attrib' structure an entry:
 *
 *	"-v <basic_vlist>,<v_value>,<current_vlist>
 *	and this 'attrib' is something that will be passed onto a
 *	PBS job before submission.
 *
 * @param[in]	basic_vlist - the basic variables list string of job.
 * @param[in]	curent_envlist - the variables list
 *		string representing the environment where qsub was
 *		invoked.
 *
 * @return	boolean (int)
 * @retval	TRUE for success.
 * @retval	FALSE for failure.
 *
 */
static int
set_job_env(char *basic_vlist, char *current_vlist)
{
	char *job_env;
	int len;

	char *s, *c, *env, l, *pc;

	/* Calculate how big to make the variable string. */
	len = 0;
	if (v_opt)
		len += strlen(v_value);

	if ((basic_vlist == NULL) || (basic_vlist[0] == '\0'))
		return FALSE;

	len += strlen(basic_vlist);

	if (V_opt && (current_vlist != NULL) && (current_vlist[0] != '\0'))
		len += strlen(current_vlist);

	len += len; /* Double it for all the commas, etc. */
	if ((job_env = (char *) malloc(len)) == NULL)
		return FALSE;
	*job_env = '\0';

	pbs_strncpy(job_env, basic_vlist, len);

	/* Send these variables with the job. */
	/* POSIX requirement: If a variable is given without a value, supply the
	 value from the environment. */
	/* MY requirement: There can be no white space in -v value. */
	if (v_opt) {
		c = v_value;
	state1: /* Initial state comes here */
		switch (*c) {
			case ',':
			case '=':
				free(job_env);
				return FALSE;
			case '\0':
				goto final;
		}
		s = c;
	state2: /* Variable name */
		switch (*c) {
			case ',':
			case '\0':
				goto state3;
			case '=':
				goto state4;
			default:
				c++;
				goto state2;
		}
	state3: /* No value - get it from qsub environment */

		/* From state3, goes back to state1, using 'c' as input */
		l = *c;
		*c = '\0';
		if (strncmp(s, PBS_O_ENV, sizeof(PBS_O_ENV) - 1) != 0) {
			/* do not add PBS_O_* env variables, as these are set by qsub */

			env = getenv(s);
			if (env == NULL) {
				free(job_env);
				return FALSE;
			}

			strcat(job_env, ",");
			strcat(job_env, s);
			strcat(job_env, "=");
			fix_path(env, 1);
			if (copy_env_value(job_env, env, 1) == NULL) {
				free(job_env);
				return FALSE;
			}
		}

		if (l == ',')
			c++;
		goto state1;
	state4: /* Value specified */

		/* From state4, goes back to state1, using 'c' as input */
		*c++ = '\0';
		;
		if (v_opt && Forwardx11_opt) {
			if (strcmp(s, "DISPLAY") == 0) {
				x11_disp = TRUE;
				free(job_env);
				return FALSE;
			}
		}
		pc = job_env + strlen(job_env);
		(void) strcat(job_env, ",");
		(void) strcat(job_env, s);
		(void) strcat(job_env, "=");
		fix_path(c, 1);
		if ((c = copy_env_value(job_env, c, 0)) == NULL) {
			free(job_env);
			return FALSE;
		}

		/* Have to undo here, since 'c' was incremented by copy_env_value */
		if (strncmp(s, PBS_O_ENV, sizeof(PBS_O_ENV) - 1) == 0)
			/* ignore PBS_O_ env variables as these are created by qsub */
			*pc = '\0';

		goto state1;
	}

final:

	if (V_opt && (current_vlist != NULL) && (current_vlist[0] != '\0'))
		/* Send every environment variable with the job. */
		strcat(job_env, current_vlist);

	set_attr_error_exit(&attrib, ATTR_v, job_env);
	free(job_env);

	return TRUE;
}

/* End of "Environment Variables" functions. */

/* The following functions support the "Daemon" capability of qsub. */

/*
 * static buffer and length used by various messages for communication
 * between the qsub foreground and background process
 */
static char *daemon_buf = NULL;
static int daemon_buflen = 0;

/**
 * @brief
 *  Resize the static variable daemon_buf.
 *
 * @param bufused - Amount of the buffer already used
 * @param lenreq - Amount of length required by new data
 *
 * @return - Error code
 * @retval - 0 - Success
 * @retval - -1 - Error
 *
 */
static int
resize_daemon_buf(int bufused, int lenreq)
{
	char *p;
	int new_buflen = lenreq + bufused;

	if (daemon_buflen < new_buflen) {
		new_buflen += 1000; /* adding 1000 so that we realloc fewer times */
		p = realloc(daemon_buf, new_buflen);
		if (p == NULL) {
			free(daemon_buf);
			daemon_buf = NULL;
			daemon_buflen = 0;
			return -1;
		}
		daemon_buf = p;
		daemon_buflen = new_buflen;
	}
	return 0;
}

/**
 * @brief
 *	Send the attrl list to the background qsub process. This is the
 * 	attribute list that was created by the foreground process based on
 *	the options that the user has provided to qsub.
 *
 * @param[in]	s - pointer to the windows PIPE or Unix domain socket
 * @parma[in]	attrib - List of attributes created by foreground qsub process
 *
 * @return int
 * @retval	-1 - Failure
 * @retval	 0 - Success
 *
 */
int
send_attrl(void *s, struct attrl *attrib)
{
	int bufused = 0;
	int len_n = 0, len_r = 0, len_v = 0;
	char *p;
	int lenreq = 0;

	while (attrib) {
		len_n = strlen(attrib->name) + 1;
		if (attrib->resource)
			len_r = strlen(attrib->resource) + 1;
		else
			len_r = 0;
		len_v = strlen(attrib->value) + 1;

		lenreq = len_n + len_r + len_v + 3 * sizeof(int);
		if (resize_daemon_buf(bufused, lenreq) != 0)
			return -1;

		/* write the lengths */
		p = daemon_buf + bufused;
		memmove(p, &len_n, sizeof(int));
		p += sizeof(int);
		memmove(p, &len_r, sizeof(int));
		p += sizeof(int);
		memmove(p, &len_v, sizeof(int));
		p += sizeof(int);

		/* now add the strings */
		memmove(p, attrib->name, len_n);
		p += len_n;
		if (len_r > 0) {
			memmove(p, attrib->resource, len_r);
			p += len_r;
		}
		memmove(p, attrib->value, len_v);
		p += len_v;

		bufused += lenreq;

		attrib = attrib->next;
	}
	if ((dosend(s, (char *) &bufused, sizeof(int)) != 0) ||
	    (dosend(s, daemon_buf, bufused) != 0))
		return -1;

	return 0;
}

/**
 * @brief
 * 	Send a null terminated string to the peer process. Used by backrgound and
 * 	foreground qsub processes to communicate error-strings, job-ids etc.
 *
 * @param[in]	s - pointer to the windows PIPE or Unix domain socket
 * @parma[in]	str - null terminated string to send
 *
 * @return int
 * @retval	-1 - Failure
 * @retval	 0 - Success
 *
 */
int
send_string(void *s, char *str)
{
	int len = strlen(str) + 1;

	if ((dosend(s, (char *) &len, sizeof(int)) != 0) ||
	    (dosend(s, str, len) != 0))
		return -1;

	return 0;
}

/**
 * @brief
 *	Recv the attrl list from the foreground qsub process. This is the
 * 	attribute list that was created by the foreground process based on
 * 	the options that the user has provided to qsub.
 *
 * @param[in]	s - pointer to the windows PIPE or Unix domain socket
 * @parma[in]	attrib - List of attributes created by foreground qsub process
 *
 * @return int
 * @retval	-1 - Failure
 * @retval	 0 - Success
 *
 */
int
recv_attrl(void *s, struct attrl **attrib)
{
	int recvlen = 0;
	struct attrl *attr = NULL;
	char *p;
	int len_n = 0, len_r = 0, len_v = 0;
	char *attr_v_val = NULL;

	if (dorecv(s, (char *) &recvlen, sizeof(int)) != 0)
		return -1;
	if (resize_daemon_buf(0, recvlen) != 0)
		return -1;

	if (dorecv(s, daemon_buf, recvlen) != 0)
		return -1;

	p = daemon_buf;
	while (p - daemon_buf < recvlen) {
		memmove(&len_n, p, sizeof(int));
		p += sizeof(int);
		memmove(&len_r, p, sizeof(int));
		p += sizeof(int);
		memmove(&len_v, p, sizeof(int));
		p += sizeof(int);

		if (len_r > 0) {
			/* strings have null character also in daemon_buf */
			set_attr_resc_error_exit(&attr, p,
						 p + len_n,
						 p + len_n + len_r);
		} else {
			/*
			 * if value is ATTR_v, we need to add PBS_O_HOSTNAME to it
			 * Since determininig PBS_O_HOSTNAME is expensive, we do it
			 * once in the background qsub, and add it to the list that comes
			 * from the front end qsub
			 */
			if (strcmp(p, ATTR_v) == 0 && pbs_hostvar != NULL) {
				int attr_v_len = len_v + strlen(pbs_hostvar) + 1;
				attr_v_val = malloc(attr_v_len);
				if (!attr_v_val)
					return -1;
				strcpy(attr_v_val, p + len_n);
				strcat(attr_v_val, pbs_hostvar);
				set_attr_error_exit(&attr, p, attr_v_val);
				free(attr_v_val);
			} else {
				set_attr_error_exit(&attr, p, p + len_n);
			}
		}
		p += len_n + len_r + len_v;
	}
	*attrib = attr;
	return 0;
}

/**
 * @brief
 *  Recv a null terminated string from the peer process. Used by backrgound and
 * 	foreground qsub processes to communicate error-strings, job-ids etc.
 *
 * @param[in]	s - pointer to the windows PIPE or Unix domain socket
 * @parma[in]	str - null terminated string to send
 *
 * @return int
 * @retval	-1 - Failure
 * @retval	 0 - Success
 *
 */
int
recv_string(void *s, char *str)
{
	int len = 0;

	if ((dorecv(s, (char *) &len, sizeof(int)) != 0) ||
	    (dorecv(s, str, len) != 0))
		return -1;

	return 0;
}

/**
 * @brief
 *  Recv a null terminated string from the peer process. Used by background and
 * 	foreground qsub processes to communicate error-strings, job-ids etc.
 * 	This is like recv_string() except the 'strp' parameter will hold a pointer
 * 	to a newly-malloced string holding the resultant string.
 *
 * @param[in]	s - pointer to the windows PIPE or Unix domain socket
 * @parma[out]	strp - holds a pointer to the newly-malloced string.
 *
 * @return int
 * @retval	-1 - Failure
 * @retval	 0 - Success
 *
 */
int
recv_dyn_string(void *s, char **strp)
{
	int recvlen = 0;

	if (dorecv(s, (char *) &recvlen, sizeof(int)) != 0)
		return -1;
	/* resizes the global 'daemon_buf' array */
	if (resize_daemon_buf(0, recvlen) != 0)
		return -1;
	if (dorecv(s, daemon_buf, recvlen) != 0)
		return -1;

	*strp = strdup(daemon_buf);
	if (*strp == NULL)
		return -1;
	return 0;
}

/**
 * @brief
 *	Send the cmd opt values for each parameter supported by qsub to the
 *	background qsub process.
 *
 * @param[in]	s - pointer to the windows PIPE or Unix domain socket
 *
 * @return int
 * @retval	-1 - Failure
 * @retval	 0 - Success
 *
 */
int
send_opts(void *s)
{
	/*
	 * we are allocating a fixed size of 100. This is because we know that
	 * the list of opts to send is going to fit within 100. Specifically, for each
	 * opt we need 2 characters, and currently we have 35 opts.
	 * If a new set of opts are added, the buffer space of 100 allocated here
	 * needs to be double checked.
	 */
	if (resize_daemon_buf(0, 100) != 0)
		return -1;

	sprintf(daemon_buf,
		"%d %d %d %d %d %d %d %d %d %d "
		"%d %d %d %d %d %d %d %d %d %d "
		"%d %d %d %d %d %d %d %d %d %d "
		"%d %d %d %d %d %d ",
		a_opt, c_opt, e_opt, h_opt, j_opt,
		k_opt, l_opt, m_opt, o_opt, p_opt,
		q_opt, r_opt, u_opt, v_opt, z_opt,
		A_opt, C_opt, J_opt, M_opt, N_opt,
		S_opt, V_opt, Depend_opt, Interact_opt, Stagein_opt,
		Stageout_opt, Sandbox_opt, Grouplist_opt, Resvstart_opt,
		Resvend_opt, pwd_opt, cred_opt, block_opt, P_opt,
		relnodes_on_stageout_opt, tolerate_node_failures_opt);

	return (send_string(s, daemon_buf));
}

/**
 * @brief
 *	Recv the cmd opt values for each parameter supported by qsub from the
 *	foreground qsub process.
 *
 * @param[in]	s - pointer to the windows PIPE or Unix domain socket
 *
 * @return int
 * @retval	-1 - Failure
 * @retval	 0 - Success
 *
 */
int
recv_opts(void *s)
{
	/*
	 * we are allocating a fixed size of 100. This is because we know that
	 * the list of opts to send is going to fit within 100. Specifically, for each
	 * opt we need 2 characters, and currently we have 35 opts.
	 * If a new set of opts are added, the buffer space of 100 allocated here
	 * needs to be double checked.
	 */
	if (resize_daemon_buf(0, 100) != 0)
		return -1;

	if (recv_string(s, daemon_buf) != 0)
		return -1;

	sscanf(daemon_buf,
	       "%d %d %d %d %d %d %d %d %d %d "
	       "%d %d %d %d %d %d %d %d %d %d "
	       "%d %d %d %d %d %d %d %d %d %d "
	       "%d %d %d %d %d %d ",
	       &a_opt, &c_opt, &e_opt, &h_opt, &j_opt,
	       &k_opt, &l_opt, &m_opt, &o_opt, &p_opt,
	       &q_opt, &r_opt, &u_opt, &v_opt, &z_opt,
	       &A_opt, &C_opt, &J_opt, &M_opt, &N_opt,
	       &S_opt, &V_opt, &Depend_opt, &Interact_opt, &Stagein_opt,
	       &Stageout_opt, &Sandbox_opt, &Grouplist_opt, &Resvstart_opt,
	       &Resvend_opt, &pwd_opt, &cred_opt, &block_opt, &P_opt,
	       &relnodes_on_stageout_opt, &tolerate_node_failures_opt);
	return 0;
}

/**
 * @brief
 *	Handles the attribute errors listed from the ECL layer
 *	by iterating through the err_list parameter. It then
 *	compares the attribute name and sets and appropriate
 *	error message in retmsg to be shown to the user.
 *
 * @param[in]	err_list - The list of attribute errors returned from
 *			the ECL verification layer
 * @param[out] retmsg - The return error message to the caller
 *			to be shown to the user
 *
 * @return int
 * @retval	-1 - Failure
 * @retval	 0 - Success
 *
 */
static int
handle_attribute_errors(struct ecl_attribute_errors *err_list, char *retmsg)
{
	struct attropl *attribute;
	char *opt;
	int i;

	for (i = 0; i < err_list->ecl_numerrors; i++) {
		attribute = err_list->ecl_attrerr[i].ecl_attribute;
		if (strcmp(attribute->name, ATTR_a) == 0)
			opt = "a";
		else if (strcmp(attribute->name, ATTR_A) == 0)
			opt = "A";
		else if (strcmp(attribute->name, ATTR_project) == 0)
			opt = "P";
		else if (strcmp(attribute->name, ATTR_c) == 0)
			opt = "c";
		else if (strcmp(attribute->name, ATTR_e) == 0)
			opt = "e";
		else if (strcmp(attribute->name, ATTR_h) == 0)
			opt = "h";
		else if (strcmp(attribute->name, ATTR_inter) == 0)
			opt = "I";
		else if (strcmp(attribute->name, ATTR_j) == 0)
			opt = "j";
		else if (strcmp(attribute->name, ATTR_J) == 0)
			opt = "J";
		else if (strcmp(attribute->name, ATTR_k) == 0)
			opt = "k";
		else if (strcmp(attribute->name, ATTR_l) == 0)
			opt = "l";
		else if (strcmp(attribute->name, ATTR_m) == 0)
			opt = "m";
		else if (strcmp(attribute->name, ATTR_M) == 0)
			opt = "M";
		else if (strcmp(attribute->name, ATTR_N) == 0)
			opt = "N";
		else if (strcmp(attribute->name, ATTR_o) == 0)
			opt = "o";
		else if (strcmp(attribute->name, ATTR_p) == 0)
			opt = "p";
		else if (strcmp(attribute->name, ATTR_r) == 0)
			opt = "r";
		else if (strcmp(attribute->name, ATTR_R) == 0)
			opt = "R";
		else if (strcmp(attribute->name, ATTR_S) == 0)
			opt = "S";
		else if (strcmp(attribute->name, ATTR_u) == 0)
			opt = "u";
		else if ((strcmp(attribute->name, ATTR_depend) == 0) ||
			 (strcmp(attribute->name, ATTR_stagein) == 0) ||
			 (strcmp(attribute->name, ATTR_stageout) == 0) ||
			 (strcmp(attribute->name, ATTR_sandbox) == 0) ||
			 (strcmp(attribute->name, ATTR_g) == 0) ||
			 (strcmp(attribute->name, ATTR_inter) == 0) ||
			 (strcmp(attribute->name, ATTR_block) == 0) ||
			 (strcmp(attribute->name, ATTR_relnodes_on_stageout) == 0) ||
			 (strcmp(attribute->name, ATTR_tolerate_node_failures) == 0) ||
			 (strcmp(attribute->name, ATTR_resv_start) == 0) ||
			 (strcmp(attribute->name, ATTR_resv_end) == 0) ||
			 (strcmp(attribute->name, ATTR_umask) == 0) ||
			 (strcmp(attribute->name, ATTR_runcount) == 0) ||
			 (strcmp(attribute->name, ATTR_cred) == 0))
			opt = "W";
		else
			return 0;

		if (*opt == 'l') {
			sprintf(retmsg, "qsub: %s\n",
				err_list->ecl_attrerr[i].ecl_errmsg);
			return (err_list->ecl_attrerr[i].ecl_errcode);
		} else if (err_list->ecl_attrerr->ecl_errcode == PBSE_JOBNBIG) {
			sprintf(retmsg, "qsub: Job %s \n", err_list->ecl_attrerr->ecl_errmsg);
			return (2);
		} else {
			sprintf(retmsg, "qsub: illegal -%s value\n", opt);
			return (2);
		}
	}
	return 0;
}

/**
 * @brief
 *	This functions connects to the pbs_server.
 *
 * @param[in] server_out - The target server name, if any, else NULL
 * @param[out] retmsg	 - Any error string is returned in this parameter
 *
 * @return int
 * @retval 0 - Success
 * @retval 1/pbs_errno - Failure, retmsg paramter is set
 *
 */
int
do_connect(char *server_out, char *retmsg)
{
	int rc = 0;
	char host[PBS_MAXHOSTNAME + 1];

	/* Set single threaded mode */
	pbs_client_thread_set_single_threaded_mode();

	/* Perform needed security library initializations (including none) */
	if (CS_client_init() == CS_SUCCESS) {
		cs_init = 1;
	} else {
		sprintf(retmsg, "qsub: unable to initialize security library.\n");
		return 1;
	}

	/* Connect to the server */
	if ((Interact_opt == FALSE) && (block_opt == FALSE))
		sd_svr = cnt2server_extend(server_out, QSUB_DAEMON);
	else
		sd_svr = cnt2server(server_out);

	if (sd_svr <= 0) {
		sprintf(retmsg, "qsub: cannot connect to server %s (errno=%d)\n", pbs_server, pbs_errno);
		return (pbs_errno);
	}

	refresh_dfltqsubargs();

	pbs_hostvar = malloc(pbs_o_hostsize + PBS_MAXHOSTNAME + 1);
	if (!pbs_hostvar) {
		sprintf(retmsg, "qsub: out of memory\n");
		return (2);
	}
	if ((rc = gethostname(host, (sizeof(host) - 1))) == 0) {
		if ((rc = get_fullhostname(host, host, (sizeof(host) - 1))) == 0) {
			snprintf(pbs_hostvar, pbs_o_hostsize + PBS_MAXHOSTNAME + 1, ",PBS_O_HOST=%s", host);
		}
	}
	if (rc != 0) {
		sprintf(retmsg, "qsub: cannot get full local host name\n");
		return (3);
	}
	return 0;
}

/**
 * @brief
 *	This functions does a job submission to the server using the global
 *	connected server socket sd_svr.
 *
 * @param[out] retmsg	 - Any error string is returned in this parameter
 *
 * @return int
 * @retval 0 - Success
 * @retval 1/-1/pbs_errno - Failure, retmsg paramter is set
 * @retval DMN_REFUSE_EXIT - If daemon can't submit the job
 *
 */
static int
do_submit(char *retmsg)
{
	struct ecl_attribute_errors *err_list;
	char *new_jobname = NULL;
	int rc;
	char *errmsg;
	int retries;

	if (dfltqsubargs != NULL) {
		/*
		 * Setting options from the server defaults will not overwrite
		 * options set from the job script. CMDLINE-2 means
		 * "one less than job script priority"
		 */
		for (retries = 2; retries > 0; retries--) {
			rc = do_dir(dfltqsubargs, CMDLINE - 2, retmsg, MAXPATHLEN);
			if (rc >= 0)
				break;
			if (retries == 2) {
				refresh_dfltqsubargs();
				if (pbs_errno != PBSE_NONE)
					return (pbs_errno);
			}
		}
		if (rc != 0)
			return (rc);
	}

	/*
	 * get environment variable if -V option is set. Return the code
	 * DMN_REFUSE_EXIT if -V option is detected in background qsub.
	 */
	if (V_opt) {
		if (is_background)
			return DMN_REFUSE_EXIT;
		qsub_envlist = env_array_to_varlist(environ);
	}

	/* set_job_env must be done here to pick up -v, -V options passed by default_qsub_arguments */
	if (!set_job_env(basic_envlist, qsub_envlist)) {
		if (x11_disp)
			snprintf(retmsg, MAXPATHLEN, "qsub: invalid usage of incompatible option –X with –v DISPLAY\n");
		else
			snprintf(retmsg, MAXPATHLEN, "qsub: cannot send environment with the job\n");
		return 1;
	}

	/* Send submit request to the server. */
	pbs_errno = 0;
	if (cred_buf) {
		/* A credential was obtained, call the credential version of submit */
		new_jobname = pbs_submit_with_cred(sd_svr, (struct attropl *) attrib,
						   script_tmp, destination, NULL, cred_type,
						   cred_len, cred_buf);
	} else {
		new_jobname = pbs_submit(sd_svr, (struct attropl *) attrib,
					 script_tmp, destination, NULL);
	}
	if (new_jobname == NULL) {

		if ((err_list = pbs_get_attributes_in_error(sd_svr))) {
			rc = handle_attribute_errors(err_list, retmsg);
			if (rc != 0)
				return rc;
		}

		errmsg = pbs_geterrmsg(sd_svr);
		if (errmsg != NULL) {
			if (strcmp(errmsg, msg_force_qsub_update) == 0)
				return PBSE_FORCE_QSUB_UPDATE;
			sprintf(retmsg, "qsub: %s\n", errmsg);
		} else {
			sprintf(retmsg, "qsub: Error (%d) submitting job\n", pbs_errno);
		}
		return (pbs_errno);
	} else {
		sprintf(retmsg, "%s", new_jobname);
		free(new_jobname);
	}
	return 0;
}

/**
 * @brief
 *	Save original values of qsub option variables.
 *
 */
static void
save_opts(void)
{
	/* save the values */
	a_opt_o = a_opt;
	c_opt_o = c_opt;
	e_opt_o = e_opt;
	h_opt_o = h_opt;
	j_opt_o = j_opt;
	k_opt_o = k_opt;
	l_opt_o = l_opt;
	m_opt_o = m_opt;
	o_opt_o = o_opt;
	p_opt_o = p_opt;
	q_opt_o = q_opt;
	r_opt_o = r_opt;
	u_opt_o = u_opt;
	v_opt_o = v_opt;
	z_opt_o = z_opt;
	A_opt_o = A_opt;
	C_opt_o = C_opt;
	J_opt_o = J_opt;
	M_opt_o = M_opt;
	N_opt_o = N_opt;
	P_opt_o = P_opt;
	S_opt_o = S_opt;
	V_opt_o = V_opt;
	Depend_opt_o = Depend_opt;
	Interact_opt_o = Interact_opt;
	Stagein_opt_o = Stagein_opt;
	Stageout_opt_o = Stageout_opt;
	Sandbox_opt_o = Sandbox_opt;
	Grouplist_opt_o = Grouplist_opt;
	gui_opt_o = gui_opt;
	Resvstart_opt_o = Resvstart_opt;
	Resvend_opt_o = Resvend_opt;
	pwd_opt_o = pwd_opt;
	cred_opt_o = cred_opt;
	block_opt_o = block_opt;
	relnodes_on_stageout_opt_o = relnodes_on_stageout_opt;
	tolerate_node_failures_opt_o = tolerate_node_failures_opt;
}

/**
 * @brief
 *	Initialize qsub option variables to their original values.
 *
 */
static void
restore_opts(void)
{
	/* save the values */
	a_opt = a_opt_o;
	c_opt = c_opt_o;
	e_opt = e_opt_o;
	h_opt = h_opt_o;
	j_opt = j_opt_o;
	k_opt = k_opt_o;
	l_opt = l_opt_o;
	m_opt = m_opt_o;
	o_opt = o_opt_o;
	p_opt = p_opt_o;
	q_opt = q_opt_o;
	r_opt = r_opt_o;
	u_opt = u_opt_o;
	v_opt = v_opt_o;
	z_opt = z_opt_o;
	A_opt = A_opt_o;
	C_opt = C_opt_o;
	J_opt = J_opt_o;
	M_opt = M_opt_o;
	N_opt = N_opt_o;
	P_opt = P_opt_o;
	S_opt = S_opt_o;
	V_opt = V_opt_o;
	Depend_opt = Depend_opt_o;
	Interact_opt = Interact_opt_o;
	Stagein_opt = Stagein_opt_o;
	Stageout_opt = Stageout_opt_o;
	Sandbox_opt = Sandbox_opt_o;
	Grouplist_opt = Grouplist_opt_o;
	Resvstart_opt = Resvstart_opt_o;
	Resvend_opt = Resvend_opt_o;
	pwd_opt = pwd_opt_o;
	cred_opt = cred_opt_o;
	block_opt = block_opt_o;
	relnodes_on_stageout_opt = relnodes_on_stageout_opt_o;
	tolerate_node_failures_opt = tolerate_node_failures_opt_o;
}

/**
 * @brief
 *	Helper function to free a list of attributes. This is called from
 *	do_daemon_stuff, since that function loops over for each client request.
 *	No freeing the list of attributes created would result in a lot of
 *	memory leak.
 *
 * @param[in]	attrib - The list of attributes to free
 *
 */
void
qsub_free_attrl(struct attrl *attrib)
{
	struct attrl *attr;

	while (attrib) {
		free(attrib->name);
		free(attrib->resource);
		free(attrib->value);

		attr = attrib;
		attrib = attrib->next;
		free(attr);
	}
}

/**
 * @brief
 *	Helper function to duplicate the passed attrl structure.
 *
 * @param[in]	attrib - The list of attributes to copy
 *
 * @return	struct attrl * - the duplicated 'attrib' (malloced).
 * @retval	non-NULL - success.
 * @retval	NULL - failure.
 *
 */
static struct attrl *
dup_attrl(struct attrl *attrib)
{
	struct attrl *attr = NULL;
	struct attrl *attr_new = NULL;

	for (attr = attrib; attr != NULL; attr = attr->next) {
		if (attr->resource != NULL) {
			/* strings have null character also in buf */
			set_attr_resc_error_exit(&attr_new, attr->name,
						 attr->resource,
						 attr->value);
		} else {
			set_attr_error_exit(&attr_new, attr->name, attr->value);
		}
	}

	return attr_new;
}

/**
 *
 * @brief
 *	The wrapper program to "do_submit()".
 * @par
 *	This attempts up to 'retry' times to do_submit(), when this function
 *	returns PBSE_FORCE_QSUB_UPDATE.
 *
 * @param[in]	retmsg - gets filled with the error message.
 *
 * @return 	int
 * @retval	the return code of do_submit().
 * @retval	if retry time exhausted or any unexpected failure,
 * 		return PBSE_PROTOCOL
 *
 */
int
do_submit2(char *rmsg)
{
	int retry; /* do a retry count to prevent infinite loop */
	int rc;

	rmsg[0] = '\0';
	/*
	 * Save the original job attributes/resources (attrib)
	 * before 'default_qsub_arguments" was applied.
	 */
	if (attrib != NULL) {
		if (attrib_o != NULL)
			qsub_free_attrl(attrib_o);
		attrib_o = dup_attrl(attrib); /* save attributes list */
		if (attrib_o == NULL) {
			snprintf(rmsg, MAXPATHLEN, "Failed to duplicate attributes list.\n");
			return PBSE_PROTOCOL;
		}
	}

	/* original v_value also needs to be saved as it gets mangled inside set_job_env() */
	if (v_value != NULL) {
		free(v_value_o);
		v_value_o = strdup(v_value);
		if (v_value_o == NULL) {
			snprintf(rmsg, MAXPATHLEN, "Failed to duplicate original -v value\n");
			return PBSE_PROTOCOL;
		}
	}

	/*
	 * Need to save original values of qsub option variables,
	 * as "reset_dfltqsubargs() below could "lose" memory
	 * of the option variable values. The values are
	 * needed in case a new "default_qsub_arguments come and
	 * gets reparsed.
	 */
	save_opts();

	rc = do_submit(rmsg);
	for (retry = 5; (rc == PBSE_FORCE_QSUB_UPDATE) && (retry > 0); retry--) {
		/* Let's retry with the new "default_qsub_arguments" */
		refresh_dfltqsubargs();
		if (pbs_errno != PBSE_NONE)
			return (pbs_errno);

		/* Use the original attrib value before the previous "default_qsub_arguments" was applied. */
		if (attrib_o != NULL) {
			if (attrib != NULL)
				qsub_free_attrl(attrib);
			attrib = dup_attrl(attrib_o);
			if (attrib == NULL) {
				snprintf(rmsg, MAXPATHLEN, "Failed to duplicate attributes list\n");
				return PBSE_PROTOCOL;
			}
		}

		/* use original -v value */
		if (v_value_o != NULL) {
			free(v_value);
			v_value = strdup(v_value_o);
			if (v_value == NULL) {
				snprintf(rmsg, MAXPATHLEN, "Failed to duplicate -v value\n");
				return PBSE_PROTOCOL;
			}
		}

		restore_opts();
		rc = do_submit(rmsg);
	}
	if (retry == 0) {
		snprintf(rmsg, MAXPATHLEN, "Retry to submit a job exhausted.\n");
		rc = PBSE_PROTOCOL;
	}
	return (rc);
}

/*
 * @brief
 *  Perform a regular submit, without the daemon.
 *
 * @param[in] daemon_up - Indicates whether daemon is running
 * @return    rc        - Error code
 */
static int
regular_submit(int daemon_up)
{
	int rc = 0;
	rc = do_connect(server_out, retmsg);
	if (rc == 0) {
		if (sd_svr != -1) {
			rc = do_submit2(retmsg);
		} else
			rc = -1;
	}
	if ((rc == 0) && !(Interact_opt != FALSE || block_opt) && (daemon_up == 0) && (no_background == 0) && !V_opt)
		do_daemon_stuff();
	return rc;
}

/* End of "Daemon" functions. */

int
main(int argc, char **argv, char **envp) /* qsub */
{
	int errflg;				 /* option error */
	static char script[MAXPATHLEN + 1] = ""; /* name of script file */
	char *q_n_out;				 /* queue part of destination */
	char *s_n_out;				 /* server part of destination */
	/* server:port to send request to */
	char *cmdargs = NULL;
	int command_flag = 0;
	int rc = 0;		   /* error code for submit */
	int do_regular_submit = 1; /* used if daemon based submit fails */
	int daemon_up = 0;
	char **argv_cpy; /* copy argv for getopt */
	int i;

	/* Set signal handlers */
	(void) set_sig_handlers();

	/*
	 * Print version info and exit, if specified with --version option.
	 * Otherwise, proceed normally.
	 */
	PRINT_VERSION_AND_EXIT(argc, argv);

	/*
	 * Identify the configured tmpdir without calling pbs_loadconf().
	 * We do not want to incur the cost of parsing the services DB.
	 */
	tmpdir = pbs_get_tmpdir();
	if (tmpdir == NULL) {
		fprintf(stderr, "qsub: Failed to load configuration parameters!\n");
		exit_qsub(2);
	}

	/*
	 * If qsub command is submitted with arguments, then capture them and
	 * encode in XML format using encode_xml_arg_list() and set the
	 * "Submit_arguments" job attribute.
	 */
	if ((argc >= 2) && (cmdargs = encode_xml_arg_list(1, argc, argv))) {
		set_attr_error_exit(&attrib, ATTR_submit_arguments, cmdargs);
		free(cmdargs);
		cmdargs = NULL;
	}

	/* Process options */
	argv_cpy = calloc(argc + 1, sizeof(char *));
	if (argv_cpy == NULL) {
		fprintf(stderr, "qsub: out of memory\n");
		exit_qsub(2);
	}
	for (i = 0; i < argc; i++) {
		argv_cpy[i] = argv[i];
	}
	argv_cpy[argc] = NULL;

	errflg = process_opts(argc, argv_cpy, CMDLINE); /* get cmd-line options */
	if (errflg || ((optind < argc) && (strcmp(argv[optind], argv_cpy[optind]) != 0))) {
		/*
		 * The arguments changed, the script and "--" must have been present.
		 * getopt will move all non-options to the end of the array. In qsub's
		 * case, it will only happen if both the "script" and "-- executable"
		 * were present in the qsub command. This is unsupported usage and
		 * should exit.
		 */
		print_usage();
		exit_qsub(2);
	}
	free(argv_cpy);
	/* Process special arguments */
	command_flag = process_special_args(argc, argv, script);
	fix_path(script, 1);

	if (command_flag == 0)
		/* Read the job script from a file or stdin */
		read_job_script(script);

	/* Enable X11 Forwarding or GUI if specified */
	enable_gui();

	/* Set option default values */
	set_opt_defaults();

	/* Parse destination string */
	server_out[0] = '\0';
	if (parse_destination_id(destination, &q_n_out, &s_n_out)) {
		fprintf(stderr, "qsub: illegally formed destination: %s\n", destination);
		(void) unlink(script_tmp);
		exit_qsub(2);
	} else if (notNULL(s_n_out)) {
		snprintf(server_out, sizeof(server_out), "%s", s_n_out);
	}

	/*
	 * Get required environment variables to be sent to the server.
	 * Must be done early here, as basic_envlist and qsub_envlist will
	 * be sent to the qsub daemon if needed.
	 */
	basic_envlist = job_env_basic();
	if (basic_envlist == NULL)
		exit_qsub(3);
	if (V_opt)
		qsub_envlist = env_array_to_varlist(envp);

	/*
	 * Disable backgrounding if we are inside another qsub
	 */
	if (getenv(ENV_PBS_JOBID) != NULL)
		no_background = 1;

	/*
	 * In case of interactive jobs, jobs with block=true, or no_background == 1,
	 * qsub should fully execute from the foreground, so daemon_submit() is not called.
	 * It should not fork, neither should it send the data to the background qsub.
	 *
	 * If all 3 of these options are zero, then try to submit via daemon.
	 */
	if ((Interact_opt || block_opt || no_background) == 0) {
		/* Try to submit jobs using a daemon */
		rc = daemon_submit(&daemon_up, &do_regular_submit);
	}

	if (do_regular_submit == 1)
		/* submission via daemon was not successful, so do regular submit */
		rc = regular_submit(daemon_up);

	/* remove temporary job script file */
	(void) unlink(script_tmp);

	if (rc == 0) { /* submit was successful */
		new_jobname = retmsg;
		if (!z_opt && Interact_opt == FALSE)
			printf("%s\n", retmsg); /* print jobid with a \n */
	} else {
		/* error, print whatever our daemon gave us back */
		fprintf(stderr, "%s", retmsg);
		/* check if the retmsg has "qsub: illegal -" string, if so print usage */
		if (strstr(retmsg, "qsub: illegal -"))
			print_usage();
		exit_qsub(rc);
	}

	/* is this an interactive job ??? */
	if (Interact_opt != FALSE)
		interactive();
	else if (block_opt) { /* block until job completes? */
		fflush(stdout);
		block();
	}

	exit_qsub(0);
	return (0);
} /* end of main() */
